---
title: 'Analysis sensitivity'
author: 'David Hume'
date: '`r format(Sys.time(), "%d %B %Y")`'
output: 
   html_document:
    toc: true
    toc_depth: 3 
editor_options: 
  chunk_output_type: console
---
---

```{r setup, message=FALSE, include = FALSE}
library(tidyverse)
library(data.table)
library(Hmisc)
library(fasttime)
library(lubridate)
library(stringr)
library(cowplot)
library(car)
library(stargazer)
library(lfe)
library(plm)
library(knitr)
library(pryr)
library(DescTools)
library(broom)
library(scales)
options(width=200, scipen=10)
knitr::opts_chunk$set(echo = TRUE, cache = FALSE, warning = FALSE, 
                      message = FALSE, cache.lazy = FALSE)

```

<div style="line-height: 2em;">
<font size="4"> 

---

```{r echo = FALSE, eval=TRUE}

Time <- Sys.time()

# Set-up

# Switches & setting of variables

# Data set either "All8pc", " "All1pc", "Test", "Render"
analysis.1 <- "Render"
# Filetype for output
# "latex" 
# "html" - inserting into html [htmltools::includeHTML()] or Word docs [Insert/[Text]/Object/Text from file]
# "text" - viewing on screen
filetype.1 <- "Render"


analysis <- ifelse(analysis.1 =="Render", readRDS("./Data/Analysis.rds"), analysis.1)
filetype <- ifelse(filetype.1 =="Render", readRDS("./Data/Filetype.rds"), filetype.1)


if (filetype=="latex") {
  out <- "tex"

  } else if (filetype=="html") {
      out <- "html"

    } else if (filetype=="text") {
      out <- "csv"

      } else {
        stop("latex, html or text")
}


Shift <- readRDS("./Data/Shift.rds")

cat("Length of peak price window (quarters) ", Shift)


# Set variables
fig = 0 # Counter for figures

```




```{r cache=FALSE, eval=TRUE, echo=FALSE}


# Data set used for plotting rules
Rules <- fread(paste0("./Data/Rules", Shift, ".csv"))
Rules <-  Rules[, date.val:=as.Date(date.val, origin ="1970-01-01")]


Peak_prices <- fread(paste0("./Data/Peak_prices.", analysis, Shift, ".csv"))
Peak_prices <- Peak_prices[,`:=`(date.val=as.Date(date.val, origin ="1970-01-01"),
                                 Peak_date_start=as.Date(Peak_date_start, origin ="1970-01-01"),
                                 New_peak_date=as.Date(New_peak_date, origin ="1970-01-01"),
                                 Peak_date_end_NA=as.Date(Peak_date_end_NA, origin ="1970-01-01"))]


Peak_plot <- function(TUI_select, xstart_peakdate, xend_peakdate, yshift) {
    # TUI_select  <- "{662017D1-AECE-41FA-A32D-0263DEBB96B1}"
    # TUI_select  <- "{90518E6A-C3D3-45C1-84F0-EB1C182F3716}"
    #Purchase_price <- Peak_prices[TUI==TUI_select,][, .(Purchase_price=min(Purchase_price), Start=min(get(xstart_peakdate)), End=max(get(xend_peakdate)))]
    PurchaseDate <-unname(head(Rules[TUI==TUI_select,.(date.val)],1))
    Peak_price_sub <- Peak_prices[TUI==TUI_select][, Purchase_date := PurchaseDate][, Segment_end:= Peak_date_start+365.25*3/4]
  
    ggplot(data = Rules[TUI==TUI_select,.(date.val, IdxPriceA)], aes(x= date.val, y=IdxPriceA))+
      geom_line(col="dodgerblue3")+
      #geom_segment(data=Peak_price_sub, aes(x = Purchase_date, y = Peak_price, xend = Segment_end, yend = Peak_price),
      #             linetype = "ff", color="grey", size=.04)+
      geom_point(data = Peak_price_sub, aes(x=Peak_date_start, y=Peak_price), col="dodgerblue3", size=2) +
      #geom_segment(data=Peak_prices[TUI==TUI_select,], aes(x = Peak_date_start, y = Peak_price+yshift, xend = date.val, yend = Peak_price+yshift), col="black",  linetype="dotted", size=0.5) +
      #geom_segment(data=Peak_prices[TUI==TUI_select,], aes(x = get(xstart_peakdate), y = Peak_price, xend = get(xend_peakdate), yend = Peak_price), col="black",  alpha =0.75) +
      #geom_segment(data=  Purchase_price, aes(x = Start, y = Purchase_price,  xend = End, yend = Purchase_price), col="green3",  alpha =0.75) +
      labs(x="", y = "Valuation (£000)")+
      scale_x_date(labels = date_format("%Y-%m")) +
      scale_y_continuous(limits = c(400000, 1700000), 
                         breaks=seq(400000, 1700000, by =400000), 
                         labels =seq(400, 1700, by =400))+
      theme_bw() + theme_classic() +
      theme(text = element_text(size=19, family="serif"),
            legend.position = "bottom",
            legend.title.align = .5 ,
            legend.text=element_text(size=19),
            panel.border = element_blank()  ,
            panel.grid = element_blank() ,
            panel.grid.minor = element_blank()) 
    
  }



Peak_price_plot <- Peak_plot("{662017D1-AECE-41FA-A32D-0263DEBB96B1}", "date.val", "New_peak_date",0)
print(Peak_price_plot)
fig = fig + 1
cat("Fig ",fig, "Illustration of peak price rule. New reference peak at the end of successful test period, Sensitivity analysis")


# Try Resold.indexed.regress.All1pcRules.csv & Resold.indexed.All1pcRules.csv Peak_prices.All1pcRules.csv
ggsave(paste0("./Plots/Peak_price_sensitivity", Shift,".png"), Peak_price_plot, dpi=600, width = 10, height = 5)
ggsave(paste0("./Plots/Peak_price_sensitivity", Shift,".png"), Peak_price_plot, dpi=600, width = 10, height = 5)

Peak_price_plot <- NULL
Peak_prices <- NULL
Rules <- NULL

```




```{r cache=FALSE, eval=TRUE, echo=FALSE}

Keep_cols <- c("Property_Id", "Type", "New", "Quality100000", "Region", "date.val", "YrQtr", "sold.dummy", "IdxPriceA", "PriceA", "DurationVal", "Peak_price", "DurationPeak")

Resold.indexed.regress <-fread(paste0("./Data/Resold.indexed.", analysis, Shift,".csv"), select = Keep_cols)

Resold.indexed.regress <- Resold.indexed.regress[,`:=`(Type=factor(Type), 
                                                         New=factor(New), 
                                                         YrQtr=factor(YrQtr),
                                                         Region=factor(Region),
                                                         Property_Id = factor(Property_Id),
                                                         date.val=as.Date(date.val, origin ="1970-01-01"),
                                                       Val_year=factor(year(date.val)),
                                                         Val_qtr=factor(quarter(date.val)))][
                                                           ,`:=`(Type = relevel(Type, "Flat"), New = relevel(New, "Old"))]

cat("DATA:", analysis)

```







```{r cache=FALSE,  echo= FALSE}



regression_variables <- function(dt, Ref_price, Ref_duration, return) {
  # NB Ref_price needs quotes
  DT <- dt[, .(Ref_price=get(Ref_price), Duration = get(Ref_duration), IdxPriceA, PriceA, Purch_duration = DurationVal, 
               sold.dummy, Property_Id, YrQtr, Val_year, Val_qtr, Type, New, Quality100000, Region)]
  
  DT[, F_unrealgain :=round(IdxPriceA-Ref_price) # Calculates unrealised gain or loss - price paid vs index
     ][, `:=`(F_unrealgainAdj = Winsorize(F_unrealgain/100000, probs = c(0.01, 0.99), na.rm = TRUE),
              F_unrealgainYN = F_unrealgain >=0,
              Price_unrealgainAdj = Winsorize((IdxPriceA-PriceA)/100000, probs = c(0.01, 0.99), na.rm = TRUE),
              Price_unrealgainYN = ifelse(IdxPriceA-PriceA>=0, TRUE, FALSE))
       ][, `:=`(Purch_duration2 = Purch_duration^2,
                         Purch_duration3 = Purch_duration^3,
                         Purch_duration4 = Purch_duration^4,
                         Purch_duration5 = Purch_duration^5)]
  
  
  
  if (return=="P") {
   DT[, `:=`(F_unrealgain.pos = ifelse(F_unrealgainYN,10000000*F_unrealgainAdj/Ref_price,0),
                       F_unrealgain.neg = ifelse(!F_unrealgainYN,10000000*F_unrealgainAdj/Ref_price,0),
                       Price_unrealgain.pos = ifelse(Price_unrealgainYN,10000000*Price_unrealgainAdj/PriceA,0),
                       Price_unrealgain.neg = ifelse(!Price_unrealgainYN,10000000*Price_unrealgainAdj/PriceA,0))]
  
    } else if (return=="L") {
        DT[, `:=`(F_unrealgain.pos = ifelse(F_unrealgainYN,F_unrealgainAdj,0),
                       F_unrealgain.neg = ifelse(!F_unrealgainYN,F_unrealgainAdj,0),
                       Price_unrealgain.pos = ifelse(Price_unrealgainYN,Price_unrealgainAdj,0),
                       Price_unrealgain.neg = ifelse(!Price_unrealgainYN,Price_unrealgainAdj,0))]
    } else if (return=="NA") {
        DT<-DT
      } else {
        stop("L, P or NA")
        }

  
  DT  <- DT[, c("Ref_price", "IdxPriceA", "PriceA", "Price_unrealgainAdj", "F_unrealgainAdj", "F_unrealgain"):= NULL]


  }



Both_ols1 <- function(dt, Ref_price, Ref_duration, title, sample, return){
  
  # title e.g. "Purchase", "Peak", "Neighbour"
  # sample e.g. "Incl3yr", "Excl3yr"
  # format short (excl property characteristics) or long
  return <-"NA"
  
  DT <- regression_variables(dt, Ref_price, Ref_duration, return)


ols_m4 <- felm(sold.dummy ~ Price_unrealgainYN + F_unrealgainYN + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 + Type + New + Quality100000 + Region| 0|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)

DT  <- DT[, c("Purch_duration","Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5") := NULL
          ][, c("Type", "New", "Quality100000", "Region"):= NULL]



ols_m3 <- felm(sold.dummy ~ Price_unrealgainYN + F_unrealgainYN |  0|0|  Property_Id + YrQtr, data =  DT, na.action=na.exclude)

ols_m1 <- felm(sold.dummy ~ Price_unrealgainYN |  0|0|  Property_Id + YrQtr, data =  DT, na.action=na.exclude)

ols_m2 <- felm(sold.dummy ~ F_unrealgainYN |  0|0|  Property_Id + YrQtr, data =  DT, na.action=na.exclude)

Covariate_Labels <- c("Gain Since Purchase = 1", "Gain Since Peak = 1", "Years From Purchase", "Detached = 1", "Semi-detached = 1","Terraced = 1","New-build = 1", "Quality (£100,000)")


table <- stargazer(ols_m1, ols_m2, ols_m3, ols_m4, no.space=TRUE, align=TRUE, digits = 4, digits.extra = 0,  dep.var.caption  = "",
            dep.var.labels = "Sale", object.names=FALSE, model.numbers=FALSE,  model.names=FALSE, column.labels = c("(1)", "(2)", "(3)", "(4)"), 
              covariate.labels=Covariate_Labels,
            add.lines = list(c("Years From Purchase Quintics", "NO", "NO", "NO", "YES"), 
                             c( "Region", "NO", "NO", "NO", "YES")),
              omit.stat=c("ser","f","adj.rsq"), omit=c("Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5", "Region"), 
              float = F, omit.table.layout = "n", column.sep.width = "-15pt", table.layout="-dc#-tas-",
              type = "latex")


table <- str_replace_all(table, "\\^", "")
table <- str_replace_all(table, "R\\$\\{2\\}\\$", "R$^{2}$")
write.table(table[11:(length(table)-2)], col.names = F, row.names = F, quote = FALSE, paste0("./Tables/LaTeX/", title, sample, analysis, return, Shift, "sen_Clean_ols.tex"))

stargazer(ols_m1, ols_m2, ols_m3, ols_m4, no.space=TRUE, align=TRUE, digits = 4, digits.extra = 0,  dep.var.caption  = "",
            dep.var.labels = "Sale", object.names=FALSE, model.numbers=FALSE,  model.names=FALSE, column.labels = c("(1)", "(2)", "(3)", "(4)"), 
              covariate.labels=Covariate_Labels,
            add.lines = list(c("Years From Purchase Quintics", "NO", "NO", "NO", "YES"), 
                             c( "Region", "NO", "NO", "NO", "YES")),
              omit.stat=c("ser","f","adj.rsq"), omit=c("Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5", "Region"), 
              font.size = "footnotesize", omit.table.layout = "n", column.sep.width = "-15pt", 
              type = filetype, out=paste0("./Tables/", title, sample, analysis, return, Shift, "sen_ols.", out))
  
}


Both_ols2 <- function(dt, Ref_price, Ref_duration, title, sample, return){
  
  # title e.g. "Purchase", "Peak", "Both"
  # sample e.g. Description if median spli must contain eith "high" or "low"
  # return whether calculated as alevel or %
  
  
  DT <- regression_variables(dt, Ref_price, Ref_duration, return)

ols_m3 <- felm(sold.dummy ~ Price_unrealgainYN + Price_unrealgain.pos + Price_unrealgain.neg + F_unrealgainYN + F_unrealgain.pos + F_unrealgain.neg  + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 + Type + New + Quality100000 + Region| 0|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)

ols_m2 <- felm(sold.dummy ~ F_unrealgainYN + F_unrealgain.pos + F_unrealgain.neg  + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 + Type + New + Quality100000 + Region| 0|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)

ols_m1 <- felm(sold.dummy ~ Price_unrealgainYN + Price_unrealgain.pos + Price_unrealgain.neg + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 + Type + New + Quality100000 + Region| 0|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)

 if (str_detect(sample, "high") | str_detect(sample, "low")){MS_table(ols_m3, sample)}
# Where applicable update median split summary table

 if (return=="P") {
Covariate_Labels <- c("Gain Since Purchase = 1", "Return Since Purchase > 0 (%)",  "Return Since Purchase < 0 (%)", "Gain Since Peak = 1", "Return Since Peak > 0 (%)",  "Return Since Peak < 0 (%)", "Years From Purchase")
  
    } else if (return=="L") {
Covariate_Labels <- c("Gain Since Purchase = 1", "Return Since Purchase > 0 (£100,000)",  "Return Since Purchase < 0 (£100,000)", "Gain Since Peak = 1", "Return Since Peak > 0 (£100,000)",  "Return Since Peak < 0 (£100,000)", "Years From Purchase")
} else {
        stop("L or P")
        }


table <- stargazer(ols_m1, ols_m2, ols_m3,  no.space=TRUE, align=TRUE, digits = 4, digits.extra = 0,  dep.var.caption  = "",
            dep.var.labels = "Sale", object.names=FALSE, model.numbers=FALSE,  model.names=FALSE, column.labels = c("(1)", "(2)", "(3)"), 
              covariate.labels=Covariate_Labels,
            add.lines = list(c("Years From Purchase Quintics", "YES", "YES", "YES"), 
                             c( "Property Characteristics", "YES", "YES", "YES")),
              omit.stat=c("ser","f","adj.rsq"), omit=c("Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5", "Type", "New", "Quality100000", "Region"), 
              float = F, omit.table.layout = "n", column.sep.width = "-15pt", table.layout="-dc#-tas-",
              type = "latex")


table <- str_replace_all(table, "\\^", "")
table <- str_replace_all(table, "R\\$\\{2\\}\\$", "R$^{2}$")
write.table(table[11:(length(table)-2)], col.names = F, row.names = F, quote = FALSE, paste0("./Tables/LaTeX/", title, sample, analysis, return, Shift, "sen_Clean_ols.tex"))
  
stargazer(ols_m1, ols_m2, ols_m3,  no.space=TRUE, align=TRUE, digits = 4, digits.extra = 0,  dep.var.caption  = "",
            dep.var.labels = "Sale", object.names=FALSE, model.numbers=FALSE,  model.names=FALSE, column.labels = c("(1)", "(2)", "(3)"), 
              covariate.labels=Covariate_Labels,
            add.lines = list(c("Years From Purchase Quintics", "YES", "YES", "YES"), 
                             c( "Property Characteristics", "YES", "YES", "YES")),
              omit.stat=c("ser","f","adj.rsq"), omit=c("Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5", "Type", "New", "Quality100000", "Region"), 
              font.size = "footnotesize", omit.table.layout = "n", column.sep.width = "-15pt", 
              type = filetype, out=paste0("./Tables/", title, sample, analysis, return, Shift, "sen_ols.", out))
    

}



Both_fe <- function(dt, Ref_price, Ref_duration, title, sample, return){
  
  # title e.g. "Purchase", "Peak", "Neighbour"
  # sample e.g. "Incl3yr", "Excl3yr"
  # format short (excl property characteristics) or long
  
  
  DT <- regression_variables(dt, Ref_price, Ref_duration, return)
  
  DT  <- DT[, c("Type", "New", "Quality100000", "Region"):= NULL]

ols_m3 <- felm(sold.dummy ~ Price_unrealgainYN + Price_unrealgain.pos + Price_unrealgain.neg + F_unrealgainYN + F_unrealgain.pos + F_unrealgain.neg  + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 | Property_Id + YrQtr|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)

ols_m2 <- felm(sold.dummy ~ F_unrealgainYN + F_unrealgain.pos + F_unrealgain.neg  + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 | Property_Id + YrQtr|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)

ols_m1 <- felm(sold.dummy ~ Price_unrealgainYN + Price_unrealgain.pos + Price_unrealgain.neg + Purch_duration + Purch_duration2 +  Purch_duration3 + Purch_duration4 + Purch_duration5 | Property_Id + YrQtr|0|  Property_Id + YrQtr , data =  DT, na.action=na.exclude)


 if (return=="P") {
Covariate_Labels <- c("Gain Since Purchase = 1", "Return Since Purchase > 0 (%)",  "Return Since Purchase < 0 (%)", "Gain Since Peak = 1", "Return Since Peak > 0 (%)",  "Return Since Peak < 0 (%)", "Years From Purchase")
  
    } else if (return=="L") {
Covariate_Labels <- c("Gain Since Purchase = 1", "Return Since Purchase > 0 (£100,000)",  "Return Since Purchase < 0 (£100,000)", "Gain Since Peak = 1", "Return Since Peak > 0 (£100,000)",  "Return Since Peak < 0 (£100,000)", "Years From Purchase")
} else {
        stop("L or P")
        }

table <- stargazer(ols_m1, ols_m2, ols_m3,  no.space=TRUE, align=TRUE, digits = 4, digits.extra = 0,  dep.var.caption  = "",
            dep.var.labels = "Sale", object.names=FALSE, model.numbers=FALSE,  model.names=FALSE, column.labels = c("(1)", "(2)", "(3)"), 
              covariate.labels=Covariate_Labels,
            add.lines = list(c("Years From Purchase Quintics", "YES", "YES", "YES"), 
                             c( "Property FE", "YES", "YES", "YES"), 
                             c( "Quarter FE", "YES", "YES", "YES")),
              omit.stat=c("ser","f","adj.rsq"), omit=c("Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5", "Type", "New", "Quality100000", "Region"), 
              float = F, omit.table.layout = "n", column.sep.width = "-15pt", table.layout="-dc#-tas-",
              type = "latex")


table <- str_replace_all(table, "\\^", "")
table <- str_replace_all(table, "R\\$\\{2\\}\\$", "R$^{2}$")
write.table(table[11:(length(table)-2)], col.names = F, row.names = F, quote = FALSE, paste0("./Tables/LaTeX/", title, sample, analysis, return, Shift, "sen_Clean_fe.tex"))
  
stargazer(ols_m1, ols_m2, ols_m3,  no.space=TRUE, align=TRUE, digits = 4, digits.extra = 0,  dep.var.caption  = "",
            dep.var.labels = "Sale", object.names=FALSE, model.numbers=FALSE,  model.names=FALSE, column.labels = c("(1)", "(2)", "(3)"), 
              covariate.labels=Covariate_Labels,
            add.lines = list(c("Years From Purchase Quintics", "YES", "YES", "YES"), 
                             c( "Property FE", "YES", "YES", "YES"), 
                             c( "Quarter FE", "YES", "YES", "YES")),
              omit.stat=c("ser","f","adj.rsq"), omit=c("Purch_duration2", "Purch_duration3", "Purch_duration4", "Purch_duration5", "Type", "New", "Quality100000", "Region"), 
              font.size = "footnotesize", omit.table.layout = "n", column.sep.width = "-15pt", 
              type = filetype, out=paste0("./Tables/", title, sample, analysis, return, Shift, "sen_fe.", out))
    

}


```



```{r cache=FALSE, eval=TRUE, echo=FALSE}

Resold.indexed.regress<-Resold.indexed.regress[!is.na(Peak_price),]
```




***
# Table 1: OLS-1 (Sign of gains)


```{r cache=FALSE, eval=TRUE, echo=FALSE}


Both_ols1(Resold.indexed.regress, "Peak_price", "DurationPeak", "Both","Sign")

```

***

# Table 2: OLS-2 (Magnitude of gains)

## Return as £100,000


```{r cache=FALSE, eval=TRUE, echo=FALSE}


Both_ols2(Resold.indexed.regress, "Peak_price", "DurationPeak", "Both", "Magnitude","L")

```




***



# Table 3: FE-1 (Magnitude of gains)

## Return as £100,000


```{r cache=FALSE, eval=TRUE, echo=FALSE}


Both_fe(Resold.indexed.regress, "Peak_price", "DurationPeak", "Both", "Magnitude","L")

```






***


```{r cache=FALSE, eval=TRUE, echo=FALSE}
Sys.time() - Time

```

